{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-02T04:54:00.231029Z",
     "start_time": "2025-05-02T04:53:57.478989Z"
    }
   },
   "source": [
    "from sympy.stats.sampling.sample_scipy import scipy\n",
    "######## Installations\n",
    "\n",
    "!pip install zep-cloud openai gdown\n",
    "!pip install scipy\n",
    "!pip"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: zep-cloud in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (2.3.1)\r\n",
      "Requirement already satisfied: openai in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (1.59.9)\r\n",
      "Requirement already satisfied: gdown in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (5.2.0)\r\n",
      "Requirement already satisfied: httpx>=0.21.2 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from zep-cloud) (0.27.2)\r\n",
      "Requirement already satisfied: pydantic>=1.9.2 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from zep-cloud) (2.9.2)\r\n",
      "Requirement already satisfied: typing_extensions>=4.0.0 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from zep-cloud) (4.11.0)\r\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from openai) (4.6.2)\r\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from openai) (1.8.0)\r\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from openai) (0.7.1)\r\n",
      "Requirement already satisfied: sniffio in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from openai) (1.3.0)\r\n",
      "Requirement already satisfied: tqdm>4 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from openai) (4.67.0)\r\n",
      "Requirement already satisfied: beautifulsoup4 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from gdown) (4.12.3)\r\n",
      "Requirement already satisfied: filelock in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from gdown) (3.16.1)\r\n",
      "Requirement already satisfied: requests[socks] in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from gdown) (2.32.3)\r\n",
      "Requirement already satisfied: idna>=2.8 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai) (3.4)\r\n",
      "Requirement already satisfied: certifi in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from httpx>=0.21.2->zep-cloud) (2024.8.30)\r\n",
      "Requirement already satisfied: httpcore==1.* in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from httpx>=0.21.2->zep-cloud) (1.0.2)\r\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.21.2->zep-cloud) (0.14.0)\r\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from pydantic>=1.9.2->zep-cloud) (0.6.0)\r\n",
      "Requirement already satisfied: pydantic-core==2.23.4 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from pydantic>=1.9.2->zep-cloud) (2.23.4)\r\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from beautifulsoup4->gdown) (2.5)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from requests[socks]->gdown) (2.0.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from requests[socks]->gdown) (2.1.0)\r\n",
      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from requests[socks]->gdown) (1.7.1)\r\n",
      "Requirement already satisfied: scipy in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (1.15.0)\r\n",
      "Requirement already satisfied: numpy<2.5,>=1.23.5 in /Users/prestonrasmussen/miniconda3/lib/python3.12/site-packages (from scipy) (1.26.4)\r\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "######## Imports\n",
    "\n",
    "import asyncio\n",
    "import pandas as pd\n",
    "import openai\n",
    "import gdown\n",
    "import json\n",
    "import tarfile\n",
    "from zep_cloud.client import AsyncZep\n",
    "from zep_cloud import Message\n",
    "from openai import AsyncOpenAI\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import scipy.stats as stats\n",
    "from datetime import datetime, timezone\n",
    "from pydantic import BaseModel, Field\n",
    "from zep_cloud import Message, EntityEdge, EntityNode\n",
    "from time import time"
   ],
   "id": "708169894cc04bc3",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "######## Download the eval dataset from github\n",
    "\n",
    "import requests\n",
    "\n",
    "url = \"https://raw.githubusercontent.com/snap-research/locomo/refs/heads/main/data/locomo10.json\"\n",
    "response = requests.get(url)\n",
    "locomo_df = pd.read_json(url)\n",
    "data = response.json()\n",
    "\n",
    "os.makedirs(\"locomo_eval/data\", exist_ok=True)\n",
    "\n",
    "with open(\"locomo_eval/data/locomo.json\", \"w\") as f:\n",
    "    json.dump(data, f, indent=2)\n",
    "\n",
    "print(\"JSON saved to ./data/locomo.json\")"
   ],
   "id": "7dea4cd5bafb5ef1",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "locomo_df.head()",
   "id": "666e1883142ac20e",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "######## Start up Zep and OpenAI clients\n",
    "load_dotenv()\n",
    "\n",
    "# , base_url=\"https://api.development.getzep.com/api/v2\"\n",
    "\n",
    "zep = AsyncZep(api_key=os.getenv(\"ZEP_API_KEY\"), base_url=\"https://api.getzep.com/api/v2\")\n",
    "oai_client = AsyncOpenAI(\n",
    "    api_key=os.getenv(\"OPENAI_API_KEY\"),\n",
    ")"
   ],
   "id": "dbf3df2c2cab2248",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "######## LongMemEval  - ingest each multi-session as its own single Zep user/session pair\n",
    "\n",
    "num_users = 10\n",
    "max_session_count = 35\n",
    "\n",
    "for user_idx in range(num_users):\n",
    "    conversation = locomo_df['conversation'].iloc[user_idx]\n",
    "    user_id = \"locomo_experiment_user_\" + str(user_idx)\n",
    "    session_id = \"locomo_experiment_session_\" + str(user_idx)\n",
    "    print(user_id)\n",
    "    \n",
    "    first_name = locomo_df['conversation'].iloc[user_idx]['speaker_a']\n",
    "    \n",
    "    try:\n",
    "        await zep.user.add(user_id=user_id, first_name=first_name)\n",
    "        await zep.memory.add_session(\n",
    "            user_id=user_id,\n",
    "            session_id=session_id,\n",
    "        )\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    # Uncomment this code to delete existing users\n",
    "    # try:\n",
    "    #     await zep.user.delete(user_id)\n",
    "    #     await zep.memory.delete(session_id)\n",
    "    # except:\n",
    "    #     pass\n",
    "    # \n",
    "    # continue\n",
    "\n",
    "    for session_idx in range(max_session_count):\n",
    "        session_key = f'session_{session_idx}'\n",
    "        print(session_key)\n",
    "        session = locomo_df['conversation'].iloc[user_idx].get(f'session_{session_idx}')\n",
    "        if session is None:\n",
    "            continue\n",
    "        \n",
    "        for msg in session:\n",
    "            # Get date\n",
    "            session_date = locomo_df['conversation'].iloc[user_idx].get(f'session_{session_idx}_date_time') + ' UTC'\n",
    "            date_format = '%I:%M %p on %d %B, %Y UTC'\n",
    "            date_string = datetime.strptime(session_date, date_format).replace(tzinfo=timezone.utc)\n",
    "            iso_date = date_string.isoformat()\n",
    "            \n",
    "            # Get speaker and role\n",
    "            speaker = msg.get('speaker')\n",
    "            role_type = 'user' if speaker == first_name else 'norole'\n",
    "            \n",
    "            \n",
    "            await zep.memory.add(session_id=session_id, messages=[Message(role=speaker, role_type=role_type, content=msg.get('text'), created_at=date_string.isoformat())])"
   ],
   "id": "7716ca7f8101e373",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-01T18:44:54.435600Z",
     "start_time": "2025-05-01T18:44:54.422572Z"
    }
   },
   "cell_type": "code",
   "source": "locomo_df['qa'].iloc[0]",
   "id": "d0a516970ac3afba",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'question': 'When did Caroline go to the LGBTQ support group?',\n",
       "  'answer': '7 May 2023',\n",
       "  'evidence': ['D1:3'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie paint a sunrise?',\n",
       "  'answer': 2022,\n",
       "  'evidence': ['D1:12'],\n",
       "  'category': 2},\n",
       " {'question': 'What fields would Caroline be likely to pursue in her educaton?',\n",
       "  'answer': 'Psychology, counseling certification',\n",
       "  'evidence': ['D1:9', 'D1:11'],\n",
       "  'category': 3},\n",
       " {'question': 'What did Caroline research?',\n",
       "  'answer': 'Adoption agencies',\n",
       "  'evidence': ['D2:8'],\n",
       "  'category': 1},\n",
       " {'question': \"What is Caroline's identity?\",\n",
       "  'answer': 'Transgender woman',\n",
       "  'evidence': ['D1:5'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie run a charity race?',\n",
       "  'answer': 'The sunday before 25 May 2023',\n",
       "  'evidence': ['D2:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When is Melanie planning on going camping?',\n",
       "  'answer': 'June 2023',\n",
       "  'evidence': ['D2:7'],\n",
       "  'category': 2},\n",
       " {'question': \"What is Caroline's relationship status?\",\n",
       "  'answer': 'Single',\n",
       "  'evidence': ['D3:13', 'D2:14'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline give a speech at a school?',\n",
       "  'answer': 'The week before 9 June 2023',\n",
       "  'evidence': ['D3:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Caroline meet up with her friends, family, and mentors?',\n",
       "  'answer': 'The week before 9 June 2023',\n",
       "  'evidence': ['D3:11'],\n",
       "  'category': 2},\n",
       " {'question': 'How long has Caroline had her current group of friends for?',\n",
       "  'answer': '4 years',\n",
       "  'evidence': ['D3:13'],\n",
       "  'category': 2},\n",
       " {'question': 'Where did Caroline move from 4 years ago?',\n",
       "  'answer': 'Sweden',\n",
       "  'evidence': ['D3:13', 'D4:3'],\n",
       "  'category': 1},\n",
       " {'question': \"How long ago was Caroline's 18th birthday?\",\n",
       "  'answer': '10 years ago',\n",
       "  'evidence': ['D4:5'],\n",
       "  'category': 2},\n",
       " {'question': 'What career path has Caroline decided to persue?',\n",
       "  'answer': 'counseling or mental health for Transgender people',\n",
       "  'evidence': ['D4:13', 'D1:11'],\n",
       "  'category': 1},\n",
       " {'question': \"Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?\",\n",
       "  'answer': 'Likely no',\n",
       "  'evidence': ['D4:15', 'D3:5'],\n",
       "  'category': 3},\n",
       " {'question': 'What activities does Melanie partake in?',\n",
       "  'answer': 'pottery, camping, painting, swimming',\n",
       "  'evidence': ['D5:4', 'D9:1', 'D1:12', 'D1:18'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie sign up for a pottery class?',\n",
       "  'answer': '2 July 2023',\n",
       "  'evidence': ['D5:4'],\n",
       "  'category': 2},\n",
       " {'question': 'When is Caroline going to the transgender conference?',\n",
       "  'answer': 'July 2023',\n",
       "  'evidence': ['D5:13'],\n",
       "  'category': 2},\n",
       " {'question': 'Where has Melanie camped?',\n",
       "  'answer': 'beach, mountains, forest',\n",
       "  'evidence': ['D6:16', 'D4:6', 'D8:32'],\n",
       "  'category': 1},\n",
       " {'question': \"What do Melanie's kids like?\",\n",
       "  'answer': 'dinosaurs, nature',\n",
       "  'evidence': ['D6:6', 'D4:8'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie go to the museum?',\n",
       "  'answer': '5 July 2023',\n",
       "  'evidence': ['D6:4'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Caroline have a picnic?',\n",
       "  'answer': 'The week before 6 July 2023',\n",
       "  'evidence': ['D6:11'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Caroline likely have Dr. Seuss books on her bookshelf?',\n",
       "  'answer': \"Yes, since she collects classic children's books\",\n",
       "  'evidence': ['D6:9'],\n",
       "  'category': 3},\n",
       " {'question': 'What books has Melanie read?',\n",
       "  'answer': '\"Nothing is Impossible\", \"Charlotte\\'s Web\"',\n",
       "  'evidence': ['D7:8', 'D6:10'],\n",
       "  'category': 1},\n",
       " {'question': 'What does Melanie do to destress?',\n",
       "  'answer': 'Running, pottery',\n",
       "  'evidence': ['D7:22', 'D5:4'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline go to the LGBTQ conference?',\n",
       "  'answer': '10 July 2023',\n",
       "  'evidence': ['D7:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie read the book \"nothing is impossible\"?',\n",
       "  'answer': 2022,\n",
       "  'evidence': ['D7:8'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Caroline pursue writing as a career option?',\n",
       "  'answer': 'LIkely no; though she likes reading, she wants to be a counselor',\n",
       "  'evidence': ['D7:5', 'D7:9'],\n",
       "  'category': 3},\n",
       " {'question': 'When did Caroline go to the adoption meeting?',\n",
       "  'answer': 'The friday before 15 July 2023',\n",
       "  'evidence': ['D8:9'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie go to the pottery workshop?',\n",
       "  'answer': 'The Friday before 15 July 2023',\n",
       "  'evidence': ['D8:2'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Melanie be considered a member of the LGBTQ community?',\n",
       "  'answer': 'Likely no, she does not refer to herself as part of it',\n",
       "  'evidence': [],\n",
       "  'category': 3},\n",
       " {'question': 'When did Melanie go camping in June?',\n",
       "  'answer': 'The week before 27 June 2023',\n",
       "  'evidence': ['D4:8'],\n",
       "  'category': 2},\n",
       " {'question': 'What LGBTQ+ events has Caroline participated in?',\n",
       "  'answer': 'Pride parade, school speech, support group',\n",
       "  'evidence': ['D5:1', 'D8:17', 'D3:1', 'D1:3'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline go to a pride parade during the summer?',\n",
       "  'answer': 'The week before 3 July 2023',\n",
       "  'evidence': ['D5:1'],\n",
       "  'category': 2},\n",
       " {'question': 'What events has Caroline participated in to help children?',\n",
       "  'answer': 'Mentoring program, school speech',\n",
       "  'evidence': ['D9:2', 'D3:3'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie go camping in July?',\n",
       "  'answer': 'two weekends before 17 July 2023',\n",
       "  'evidence': ['D9:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Caroline join a mentorship program?',\n",
       "  'answer': 'The weekend before 17 July 2023',\n",
       "  'evidence': ['D9:2'],\n",
       "  'category': 2},\n",
       " {'question': 'What did Melanie paint recently?',\n",
       "  'answer': 'sunset',\n",
       "  'evidence': ['D8:6; D9:17'],\n",
       "  'category': 1},\n",
       " {'question': 'What activities has Melanie done with her family?',\n",
       "  'answer': 'Pottery, painting, camping, museum, swimming, hiking',\n",
       "  'evidence': ['D8:4', 'D8:6', 'D9:1', 'D6:4', 'D1:18', 'D3:14'],\n",
       "  'category': 1},\n",
       " {'question': 'In what ways is Caroline participating in the LGBTQ community?',\n",
       "  'answer': 'Joining activist group, going to pride parades, participating in an art show, mentoring program',\n",
       "  'evidence': ['D10:3', 'D5:1', 'D9:12', 'D9:2'],\n",
       "  'category': 1},\n",
       " {'question': 'How many times has Melanie gone to the beach in 2023?',\n",
       "  'answer': 2,\n",
       "  'evidence': ['D10:8', 'D6:16'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline join a new activist group?',\n",
       "  'answer': 'The Tuesday before 20 July 2023',\n",
       "  'evidence': ['D10:3'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Melanie be more interested in going to a national park or a theme park?',\n",
       "  'answer': 'National park; she likes the outdoors',\n",
       "  'evidence': ['D10:12', 'D10:14'],\n",
       "  'category': 3},\n",
       " {'question': 'What kind of art does Caroline make?',\n",
       "  'answer': 'abstract art',\n",
       "  'evidence': ['D11:12', 'D11:8', 'D9:14'],\n",
       "  'category': 1},\n",
       " {'question': \"When is Melanie's daughter's birthday?\",\n",
       "  'answer': '13 August',\n",
       "  'evidence': ['D11:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Caroline attend a pride parade in August?',\n",
       "  'answer': 'The Friday before 14 August 2023',\n",
       "  'evidence': ['D11:4'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Melanie be considered an ally to the transgender community?',\n",
       "  'answer': 'Yes, she is supportive',\n",
       "  'evidence': [],\n",
       "  'category': 3},\n",
       " {'question': 'Who supports Caroline when she has a negative experience?',\n",
       "  'answer': 'Her mentors, family, and friends',\n",
       "  'evidence': ['D12:1', 'D3:11'],\n",
       "  'category': 1},\n",
       " {'question': 'What types of pottery have Melanie and her kids made?',\n",
       "  'answer': 'bowls, cup',\n",
       "  'evidence': ['D12:14', 'D8:4', 'D5:6'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline and Melanie go to a pride fesetival together?',\n",
       "  'answer': 2022,\n",
       "  'evidence': ['D12:15'],\n",
       "  'category': 2},\n",
       " {'question': \"What would Caroline's political leaning likely be?\",\n",
       "  'answer': 'Liberal',\n",
       "  'evidence': ['D12:1'],\n",
       "  'category': 3},\n",
       " {'question': 'What has Melanie painted?',\n",
       "  'answer': 'Horse, sunset, sunrise',\n",
       "  'evidence': ['D13:8', 'D8:6', 'D1:12'],\n",
       "  'category': 1},\n",
       " {'question': \"What are Melanie's pets' names?\",\n",
       "  'answer': 'Oliver, Luna, Bailey',\n",
       "  'evidence': ['D13:4', 'D7:18'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline apply to adoption agencies?',\n",
       "  'answer': 'The week of 23 August 2023',\n",
       "  'evidence': ['D13:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Caroline draw a self-portrait?',\n",
       "  'answer': 'The week before 23 August 2023',\n",
       "  'evidence': ['D13:11'],\n",
       "  'category': 2},\n",
       " {'question': 'What subject have Caroline and Melanie both painted?',\n",
       "  'answer': 'Sunsets',\n",
       "  'evidence': ['D14:5', 'D8:6'],\n",
       "  'category': 1},\n",
       " {'question': 'What symbols are important to Caroline?',\n",
       "  'answer': 'Rainbow flag, transgender symbol',\n",
       "  'evidence': ['D14:15', 'D4:1'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline encounter people on a hike and have a negative experience?',\n",
       "  'answer': 'The week before 25 August 2023',\n",
       "  'evidence': ['D14:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie make a plate in pottery class?',\n",
       "  'answer': '24 August 2023',\n",
       "  'evidence': ['D14:4'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Caroline be considered religious?',\n",
       "  'answer': 'Somewhat, but not extremely religious',\n",
       "  'evidence': ['D14:19', 'D12:1'],\n",
       "  'category': 3},\n",
       " {'question': 'What instruments does Melanie play?',\n",
       "  'answer': 'clarinet and violin',\n",
       "  'evidence': ['D15:26', 'D2:5'],\n",
       "  'category': 1},\n",
       " {'question': 'What musical artists/bands has Melanie seen?',\n",
       "  'answer': 'Summer Sounds, Matt Patterson',\n",
       "  'evidence': ['D15:16', 'D11:3'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie go to the park?',\n",
       "  'answer': '27 August 2023',\n",
       "  'evidence': ['D15:2'],\n",
       "  'category': 2},\n",
       " {'question': \"When is Caroline's youth center putting on a talent show?\",\n",
       "  'answer': 'September 2023',\n",
       "  'evidence': ['D15:11'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?',\n",
       "  'answer': \"Yes; it's classical music\",\n",
       "  'evidence': ['D15:28'],\n",
       "  'category': 3},\n",
       " {'question': 'What are some changes Caroline has faced during her transition journey?',\n",
       "  'answer': 'Changes to her body, losing unsupportive friends',\n",
       "  'evidence': ['D16:15', 'D11:14'],\n",
       "  'category': 1},\n",
       " {'question': 'What does Melanie do with her family on hikes?',\n",
       "  'answer': 'Roast marshmallows, tell stories',\n",
       "  'evidence': ['D16:4', 'D10:12'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline go biking with friends?',\n",
       "  'answer': 'The weekend before 13 September 2023',\n",
       "  'evidence': ['D16:1'],\n",
       "  'category': 2},\n",
       " {'question': 'How long has Melanie been practicing art?',\n",
       "  'answer': 'Since 2016',\n",
       "  'evidence': ['D16:8'],\n",
       "  'category': 2},\n",
       " {'question': 'What personality traits might Melanie say Caroline has?',\n",
       "  'answer': 'Thoughtful, authentic, driven',\n",
       "  'evidence': ['D16:18', 'D13:16', 'D7:4'],\n",
       "  'category': 3},\n",
       " {'question': 'What transgender-specific events has Caroline attended?',\n",
       "  'answer': 'Poetry reading, conference',\n",
       "  'evidence': ['D17:19', 'D15:13'],\n",
       "  'category': 1},\n",
       " {'question': \"What book did Melanie read from Caroline's suggestion?\",\n",
       "  'answer': '\"Becoming Nicole\"',\n",
       "  'evidence': ['D7:11', 'D17:10'],\n",
       "  'category': 1},\n",
       " {'question': \"When did Melanie's friend adopt a child?\",\n",
       "  'answer': 2022,\n",
       "  'evidence': ['D17:3'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie get hurt?',\n",
       "  'answer': 'September 2023',\n",
       "  'evidence': ['D17:8'],\n",
       "  'category': 2},\n",
       " {'question': \"When did Melanie's family go on a roadtrip?\",\n",
       "  'answer': 'The weekend before 20 October 2023',\n",
       "  'evidence': ['D18:1'],\n",
       "  'category': 2},\n",
       " {'question': 'How many children does Melanie have?',\n",
       "  'answer': 3,\n",
       "  'evidence': ['D18:1', 'D18:7'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Melanie go on a hike after the roadtrip?',\n",
       "  'answer': '19 October 2023',\n",
       "  'evidence': ['D18:17'],\n",
       "  'category': 1},\n",
       " {'question': 'Would Melanie go on another roadtrip soon?',\n",
       "  'answer': 'Likely no; since this one went badly',\n",
       "  'evidence': ['D18:3', 'D18:1'],\n",
       "  'category': 3},\n",
       " {'question': 'What items has Melanie bought?',\n",
       "  'answer': 'Figurines, shoes',\n",
       "  'evidence': ['D19:2', 'D7:18'],\n",
       "  'category': 1},\n",
       " {'question': 'When did Caroline pass the adoption interview?',\n",
       "  'answer': 'The Friday before 22 October 2023',\n",
       "  'evidence': ['D19:1'],\n",
       "  'category': 2},\n",
       " {'question': 'When did Melanie buy the figurines?',\n",
       "  'answer': '21 October 2023',\n",
       "  'evidence': ['D19:2'],\n",
       "  'category': 2},\n",
       " {'question': 'Would Caroline want to move back to her home country soon?',\n",
       "  'answer': \"No; she's in the process of adopting children.\",\n",
       "  'evidence': ['D19:1', 'D19:3'],\n",
       "  'category': 3},\n",
       " {'question': 'What did the charity race raise awareness for?',\n",
       "  'answer': 'mental health',\n",
       "  'evidence': ['D2:2'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Melanie realize after the charity race?',\n",
       "  'answer': 'self-care is important',\n",
       "  'evidence': ['D2:3'],\n",
       "  'category': 4},\n",
       " {'question': 'How does Melanie prioritize self-care?',\n",
       "  'answer': 'by carving out some me-time each day for activities like running, reading, or playing the violin',\n",
       "  'evidence': ['D2:5'],\n",
       "  'category': 4},\n",
       " {'question': \"What are Caroline's plans for the summer?\",\n",
       "  'answer': 'researching adoption agencies',\n",
       "  'evidence': ['D2:8'],\n",
       "  'category': 4},\n",
       " {'question': 'What type of individuals does the adoption agency Caroline is considering support?',\n",
       "  'answer': 'LGBTQ+ individuals',\n",
       "  'evidence': ['D2:12'],\n",
       "  'category': 4},\n",
       " {'question': 'Why did Caroline choose the adoption agency?',\n",
       "  'answer': 'because of their inclusivity and support for LGBTQ+ individuals',\n",
       "  'evidence': ['D2:12'],\n",
       "  'category': 4},\n",
       " {'question': 'What is Caroline excited about in the adoption process?',\n",
       "  'answer': 'creating a family for kids who need one',\n",
       "  'evidence': ['D2:14'],\n",
       "  'category': 4},\n",
       " {'question': \"What does Melanie think about Caroline's decision to adopt?\",\n",
       "  'answer': 'she thinks Caroline is doing something amazing and will be an awesome mom',\n",
       "  'evidence': ['D2:15'],\n",
       "  'category': 4},\n",
       " {'question': 'How long have Mel and her husband been married?',\n",
       "  'answer': 'Mel and her husband have been married for 5 years.',\n",
       "  'evidence': ['D3:16'],\n",
       "  'category': 4},\n",
       " {'question': \"What does Caroline's necklace symbolize?\",\n",
       "  'answer': 'love, faith, and strength',\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 4},\n",
       " {'question': \"What country is Caroline's grandma from?\",\n",
       "  'answer': 'Sweden',\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 4},\n",
       " {'question': \"What was grandma's gift to Caroline?\",\n",
       "  'answer': 'necklace',\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 4},\n",
       " {'question': \"What is Melanie's hand-painted bowl a reminder of?\",\n",
       "  'answer': 'art and self-expression',\n",
       "  'evidence': ['D4:5'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Melanie and her family do while camping?',\n",
       "  'answer': 'explored nature, roasted marshmallows, and went on a hike',\n",
       "  'evidence': ['D4:8'],\n",
       "  'category': 4},\n",
       " {'question': 'What kind of counseling and mental health services is Caroline interested in pursuing?',\n",
       "  'answer': 'working with trans people, helping them accept themselves and supporting their mental health',\n",
       "  'evidence': ['D4:13'],\n",
       "  'category': 4},\n",
       " {'question': 'What workshop did Caroline attend recently?',\n",
       "  'answer': 'LGBTQ+ counseling workshop',\n",
       "  'evidence': ['D4:13'],\n",
       "  'category': 4},\n",
       " {'question': 'What was discussed in the LGBTQ+ counseling workshop?',\n",
       "  'answer': 'therapeutic methods and how to best work with trans people',\n",
       "  'evidence': ['D4:13'],\n",
       "  'category': 4},\n",
       " {'question': 'What motivated Caroline to pursue counseling?',\n",
       "  'answer': 'her own journey and the support she received, and how counseling improved her life',\n",
       "  'evidence': ['D4:15'],\n",
       "  'category': 4},\n",
       " {'question': 'What kind of place does Caroline want to create for people?',\n",
       "  'answer': 'a safe and inviting place for people to grow',\n",
       "  'evidence': ['D4:15'],\n",
       "  'category': 4},\n",
       " {'question': 'Did Melanie make the black and white bowl in the photo?',\n",
       "  'answer': 'Yes',\n",
       "  'evidence': ['D5:8'],\n",
       "  'category': 4},\n",
       " {'question': 'What kind of books does Caroline have in her library?',\n",
       "  'answer': \"kids' books - classics, stories from different cultures, educational books\",\n",
       "  'evidence': ['D6:9'],\n",
       "  'category': 4},\n",
       " {'question': \"What was Melanie's favorite book from her childhood?\",\n",
       "  'answer': '\"Charlotte\\'s Web\"',\n",
       "  'evidence': ['D6:10'],\n",
       "  'category': 4},\n",
       " {'question': 'What book did Caroline recommend to Melanie?',\n",
       "  'answer': '\"Becoming Nicole\"',\n",
       "  'evidence': ['D7:11'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Caroline take away from the book \"Becoming Nicole\"?',\n",
       "  'answer': 'Lessons on self-acceptance and finding support',\n",
       "  'evidence': ['D7:13'],\n",
       "  'category': 4},\n",
       " {'question': 'What are the new shoes that Melanie got used for?',\n",
       "  'answer': 'Running',\n",
       "  'evidence': ['D7:19'],\n",
       "  'category': 4},\n",
       " {'question': \"What is Melanie's reason for getting into running?\",\n",
       "  'answer': 'To de-stress and clear her mind',\n",
       "  'evidence': ['D7:21'],\n",
       "  'category': 4},\n",
       " {'question': 'What does Melanie say running has been great for?',\n",
       "  'answer': 'Her mental health',\n",
       "  'evidence': ['D7:24'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Mel and her kids make during the pottery workshop?',\n",
       "  'answer': 'pots',\n",
       "  'evidence': ['D8:2'],\n",
       "  'category': 4},\n",
       " {'question': 'What kind of pot did Mel and her kids make with clay?',\n",
       "  'answer': 'a cup with a dog face on it',\n",
       "  'evidence': ['D8:4'],\n",
       "  'category': 4},\n",
       " {'question': 'What creative project do Mel and her kids do together besides pottery?',\n",
       "  'answer': 'painting',\n",
       "  'evidence': ['D8:5'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Mel and her kids paint in their latest project in July 2023?',\n",
       "  'answer': 'a sunset with a palm tree',\n",
       "  'evidence': ['D8:6'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Caroline see at the council meeting for adoption?',\n",
       "  'answer': 'many people wanting to create loving homes for children in need',\n",
       "  'evidence': ['D8:9'],\n",
       "  'category': 4},\n",
       " {'question': 'What do sunflowers represent according to Caroline?',\n",
       "  'answer': 'warmth and happiness',\n",
       "  'evidence': ['D8:11'],\n",
       "  'category': 4},\n",
       " {'question': 'Why are flowers important to Melanie?',\n",
       "  'answer': 'They remind her to appreciate the small moments and were a part of her wedding decor',\n",
       "  'evidence': ['D8:12'],\n",
       "  'category': 4},\n",
       " {'question': \"What inspired Caroline's painting for the art show?\",\n",
       "  'answer': 'visiting an LGBTQ center and wanting to capture unity and strength',\n",
       "  'evidence': ['D9:16'],\n",
       "  'category': 4},\n",
       " {'question': 'How often does Melanie go to the beach with her kids?',\n",
       "  'answer': 'once or twice a year',\n",
       "  'evidence': ['D10:10'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Melanie and her family see during their camping trip last year?',\n",
       "  'answer': 'Perseid meteor shower',\n",
       "  'evidence': ['D10:14'],\n",
       "  'category': 4},\n",
       " {'question': 'How did Melanie feel while watching the meteor shower?',\n",
       "  'answer': 'in awe of the universe',\n",
       "  'evidence': ['D10:18'],\n",
       "  'category': 4},\n",
       " {'question': 'Whose birthday did Melanie celebrate recently?',\n",
       "  'answer': \"Melanie's daughter\",\n",
       "  'evidence': ['D11:1'],\n",
       "  'category': 4},\n",
       " {'question': \"Who performed at the concert at Melanie's daughter's birthday?\",\n",
       "  'answer': 'Matt Patterson',\n",
       "  'evidence': ['D11:3'],\n",
       "  'category': 4},\n",
       " {'question': 'Why did Melanie choose to use colors and patterns in her pottery project?',\n",
       "  'answer': 'She wanted to catch the eye and make people smile.',\n",
       "  'evidence': ['D12:6'],\n",
       "  'category': 4},\n",
       " {'question': 'What pet does Caroline have?',\n",
       "  'answer': 'guinea pig',\n",
       "  'evidence': ['D13:3'],\n",
       "  'category': 4},\n",
       " {'question': 'What pets does Melanie have?',\n",
       "  'answer': 'Two cats and a dog',\n",
       "  'evidence': ['D13:4'],\n",
       "  'category': 4},\n",
       " {'question': 'Where did Oliver hide his bone once?',\n",
       "  'answer': \"In Melanie's slipper\",\n",
       "  'evidence': ['D13:6'],\n",
       "  'category': 4},\n",
       " {'question': 'What activity did Caroline used to do with her dad?',\n",
       "  'answer': 'Horseback riding',\n",
       "  'evidence': ['D13:7'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Caroline make for a local church?',\n",
       "  'answer': 'a stained glass window',\n",
       "  'evidence': ['D14:17'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Caroline find in her neighborhood during her walk?',\n",
       "  'answer': 'a rainbow sidewalk',\n",
       "  'evidence': ['D14:23'],\n",
       "  'category': 4},\n",
       " {'question': 'Which song motivates Caroline to be courageous?',\n",
       "  'answer': 'Brave by Sara Bareilles',\n",
       "  'evidence': ['D15:23'],\n",
       "  'category': 4},\n",
       " {'question': 'Which  classical musicians does Melanie enjoy listening to?',\n",
       "  'answer': 'Bach and Mozart',\n",
       "  'evidence': ['D15:28'],\n",
       "  'category': 4},\n",
       " {'question': 'Who is Melanie a fan of in terms of modern music?',\n",
       "  'answer': 'Ed Sheeran',\n",
       "  'evidence': ['D15:28'],\n",
       "  'category': 4},\n",
       " {'question': 'How long has Melanie been creating art?',\n",
       "  'answer': '7 years',\n",
       "  'evidence': ['D16:7'],\n",
       "  'category': 4},\n",
       " {'question': 'What precautionary sign did Melanie see at the café?',\n",
       "  'answer': 'A sign stating that someone is not being able to leave',\n",
       "  'evidence': ['D16:16'],\n",
       "  'category': 4},\n",
       " {'question': 'What advice does Caroline give for getting started with adoption?',\n",
       "  'answer': 'Do research, find an adoption agency or lawyer, gather necessary documents, and prepare emotionally.',\n",
       "  'evidence': ['D17:7'],\n",
       "  'category': 4},\n",
       " {'question': 'What setback did Melanie face in October 2023?',\n",
       "  'answer': 'She got hurt and had to take a break from pottery.',\n",
       "  'evidence': ['D17:8'],\n",
       "  'category': 4},\n",
       " {'question': 'What does Melanie do to keep herself busy during her pottery break?',\n",
       "  'answer': 'Read a book and paint.',\n",
       "  'evidence': ['D17:10'],\n",
       "  'category': 4},\n",
       " {'question': 'What painting did Melanie show to Caroline on October 13, 2023?',\n",
       "  'answer': 'A painting inspired by sunsets with a pink sky.',\n",
       "  'evidence': ['D17:12'],\n",
       "  'category': 4},\n",
       " {'question': 'What kind of painting did Caroline share with Melanie on October 13, 2023?',\n",
       "  'answer': 'An abstract painting with blue streaks on a wall.',\n",
       "  'evidence': ['D17:14'],\n",
       "  'category': 4},\n",
       " {'question': 'What was the poetry reading that Caroline attended about?',\n",
       "  'answer': 'It was a transgender poetry reading where transgender people shared their stories.',\n",
       "  'evidence': ['D17:18'],\n",
       "  'category': 4},\n",
       " {'question': 'What did the posters at the poetry reading say?',\n",
       "  'answer': '\"Trans Lives Matter\"',\n",
       "  'evidence': ['D17:19'],\n",
       "  'category': 4},\n",
       " {'question': \"What does Caroline's drawing symbolize for her?\",\n",
       "  'answer': 'Freedom and being true to herself.',\n",
       "  'evidence': ['D17:23'],\n",
       "  'category': 4},\n",
       " {'question': 'How do Melanie and Caroline describe their journey through life together?',\n",
       "  'answer': 'An ongoing adventure of learning and growing.',\n",
       "  'evidence': ['D17:25'],\n",
       "  'category': 4},\n",
       " {'question': \"What happened to Melanie's son on their road trip?\",\n",
       "  'answer': 'He got into an accident',\n",
       "  'evidence': ['D18:1'],\n",
       "  'category': 4},\n",
       " {'question': \"How did Melanie's son handle the accident?\",\n",
       "  'answer': 'He was scared but reassured by his family',\n",
       "  'evidence': ['D18:6', 'D18:7'],\n",
       "  'category': 4},\n",
       " {'question': 'How did Melanie feel about her family after the accident?',\n",
       "  'answer': 'They are important and mean the world to her',\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 4},\n",
       " {'question': \"How did Melanie's children handle the accident?\",\n",
       "  'answer': 'They were scared but resilient',\n",
       "  'evidence': ['D18:7'],\n",
       "  'category': 4},\n",
       " {'question': 'How did Melanie feel after the accident?',\n",
       "  'answer': 'Grateful and thankful for her family',\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 4},\n",
       " {'question': \"What was Melanie's reaction to her children enjoying the Grand Canyon?\",\n",
       "  'answer': 'She was happy and thankful',\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 4},\n",
       " {'question': \"What do Melanie's family give her?\",\n",
       "  'answer': 'Strength and motivation',\n",
       "  'evidence': ['D18:9'],\n",
       "  'category': 4},\n",
       " {'question': 'How did Melanie feel about her family supporting her?',\n",
       "  'answer': 'She appreciated them a lot',\n",
       "  'evidence': ['D18:13'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Melanie do after the road trip to relax?',\n",
       "  'answer': 'Went on a nature walk or hike',\n",
       "  'evidence': ['D18:17'],\n",
       "  'category': 4},\n",
       " {'question': 'What did Caroline realize after her charity race?',\n",
       "  'evidence': ['D2:3'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'self-care is important'},\n",
       " {'question': \"What are Melanie's plans for the summer with respect to adoption?\",\n",
       "  'evidence': ['D2:8'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'researching adoption agencies'},\n",
       " {'question': 'What type of individuals does the adoption agency Melanie is considering support?',\n",
       "  'evidence': ['D2:12'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'LGBTQ+ individuals'},\n",
       " {'question': 'Why did Melanie choose the adoption agency?',\n",
       "  'evidence': ['D2:12'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'because of their inclusivity and support for LGBTQ+ individuals'},\n",
       " {'question': 'What is Melanie excited about in her adoption process?',\n",
       "  'evidence': ['D2:14'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'creating a family for kids who need one'},\n",
       " {'question': \"What does Melanie's necklace symbolize?\",\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'love, faith, and strength'},\n",
       " {'question': \"What country is Melanie's grandma from?\",\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Sweden'},\n",
       " {'question': \"What was grandma's gift to Melanie?\",\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'necklace'},\n",
       " {'question': \"What was grandpa's gift to Caroline?\",\n",
       "  'evidence': ['D4:3'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'necklace'},\n",
       " {'question': \"What is Caroline's hand-painted bowl a reminder of?\",\n",
       "  'evidence': ['D4:5'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'art and self-expression'},\n",
       " {'question': 'What did Caroline and her family do while camping?',\n",
       "  'evidence': ['D4:8'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'explored nature, roasted marshmallows, and went on a hike'},\n",
       " {'question': 'What kind of counseling and mental health services is Melanie interested in pursuing?',\n",
       "  'evidence': ['D4:13'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'working with trans people, helping them accept themselves and supporting their mental health'},\n",
       " {'question': 'What kind of counseling workshop did Melanie attend recently?',\n",
       "  'evidence': ['D4:13'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'LGBTQ+ counseling workshop'},\n",
       " {'question': 'What motivated Melanie to pursue counseling?',\n",
       "  'evidence': ['D4:15'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'her own journey and the support she received, and how counseling improved her life'},\n",
       " {'question': 'What kind of place does Melanie want to create for people?',\n",
       "  'evidence': ['D4:15'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'a safe and inviting place for people to grow'},\n",
       " {'question': 'Did Caroline make the black and white bowl in the photo?',\n",
       "  'adversarial_answer': 'Yes',\n",
       "  'answer': 'No',\n",
       "  'evidence': ['D5:8'],\n",
       "  'category': 5},\n",
       " {'question': 'What are the new shoes that Caroline got used for?',\n",
       "  'evidence': ['D7:19'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Running'},\n",
       " {'question': \"What is Caroline's reason for getting into running?\",\n",
       "  'evidence': ['D7:21'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'To de-stress and clear her mind'},\n",
       " {'question': 'What does Caroline say running has been great for?',\n",
       "  'evidence': ['D7:24'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Her mental health'},\n",
       " {'question': 'What did Melanie see at the council meeting for adoption?',\n",
       "  'evidence': ['D8:9'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'many people wanting to create loving homes for children in need'},\n",
       " {'question': \"What inspired Melanie's painting for the art show?\",\n",
       "  'evidence': ['D9:16'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'visiting an LGBTQ center and wanting to capture unity and strength'},\n",
       " {'question': \"What inspired Caroline's sculpture for the art show?\",\n",
       "  'evidence': ['D9:16'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'visiting an LGBTQ center and wanting to capture unity and strength'},\n",
       " {'question': 'How often does Caroline go to the beach with her kids?',\n",
       "  'evidence': ['D10:10'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'once or twice a year'},\n",
       " {'question': 'What did Caroline and her family see during their camping trip last year?',\n",
       "  'evidence': ['D10:14'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Perseid meteor shower'},\n",
       " {'question': 'How did Caroline feel while watching the meteor shower?',\n",
       "  'evidence': ['D10:18'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'in awe of the universe'},\n",
       " {'question': 'Why did Caroline choose to use colors and patterns in her pottery project?',\n",
       "  'evidence': ['D12:6'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'She wanted to catch the eye and make people smile.'},\n",
       " {'question': \"Is Oscar Melanie's pet?\",\n",
       "  'adversarial_answer': 'Yes',\n",
       "  'answer': 'No',\n",
       "  'evidence': ['D13:3'],\n",
       "  'category': 5},\n",
       " {'question': 'Where did Oscar hide his bone once?',\n",
       "  'evidence': ['D13:6'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': \"In Melanie's slipper\"},\n",
       " {'question': 'What activity did Melanie used to do with her dad?',\n",
       "  'evidence': ['D13:7'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Horseback riding'},\n",
       " {'question': 'What did Melanie make for a local church?',\n",
       "  'evidence': ['D14:17'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'a stained glass window'},\n",
       " {'question': 'What did Melanie find in her neighborhood during her walk?',\n",
       "  'evidence': ['D14:23'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'a rainbow sidewalk'},\n",
       " {'question': 'Which song motivates Melanie to be courageous?',\n",
       "  'evidence': ['D15:23'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Brave by Sara Bareilles'},\n",
       " {'question': 'What type of instrument does Caroline play?',\n",
       "  'evidence': ['D15:26'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'clarinet and violin'},\n",
       " {'question': 'Which classical musicians does Caroline enjoy listening to?',\n",
       "  'evidence': ['D15:28'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Bach and Mozart'},\n",
       " {'question': 'Who is Caroline a fan of in terms of modern music?',\n",
       "  'evidence': ['D15:28'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Ed Sheeran'},\n",
       " {'question': 'What precautionary sign did Caroline see at the café?',\n",
       "  'evidence': ['D16:16'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'A sign stating that someone is not being able to leave'},\n",
       " {'question': 'What setback did Caroline face recently?',\n",
       "  'evidence': ['D17:8'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'She got hurt and had to take a break from pottery.'},\n",
       " {'question': 'What does Caroline do to keep herself busy during her pottery break?',\n",
       "  'evidence': ['D17:10'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Read a book and paint.'},\n",
       " {'question': 'What was the poetry reading that Melanie attended about?',\n",
       "  'evidence': ['D17:18'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'It was a transgender poetry reading where transgender people shared their stories.'},\n",
       " {'question': \"What happened to Caroline's son on their road trip?\",\n",
       "  'evidence': ['D18:1'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'He got into an accident'},\n",
       " {'question': \"How did Caroline's son handle the accident?\",\n",
       "  'evidence': ['D18:6', 'D18:7'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'He was scared but reassured by his family'},\n",
       " {'question': 'How did Caroline feel about her family after the accident?',\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'They are important and mean the world to her'},\n",
       " {'question': \"How did Caroline's children handle the accident?\",\n",
       "  'evidence': ['D18:7'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'They were scared but resilient'},\n",
       " {'question': 'How did Caroline feel after the accident?',\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Grateful and thankful for her family'},\n",
       " {'question': \"What was Caroline's reaction to her children enjoying the Grand Canyon?\",\n",
       "  'evidence': ['D18:5'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'She was happy and thankful'},\n",
       " {'question': 'What did Caroline do after the road trip to relax?',\n",
       "  'evidence': ['D18:17'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Went on a nature walk or hike'},\n",
       " {'question': 'What does Caroline love most about camping with her family?',\n",
       "  'evidence': ['D18:21'],\n",
       "  'category': 5,\n",
       "  'adversarial_answer': 'Being present and bonding with her family'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "id": "1726b4ae5c984e4d",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
